{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creates subexamples for kernel matching from Google dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sub examples need to be manually labelled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "import KernelSparse\n",
    "import sys\n",
    "sys.path.insert(0, '/home/asus/Downloads/parser/corenlp-python')\n",
    "from corenlp import StanfordCoreNLP\n",
    "\n",
    "\n",
    "CORENLP_dir = \"/home/asus/Downloads/parser/corenlp-python/stanford-corenlp-full-2015-04-20\"\n",
    "CORENLP = StanfordCoreNLP(CORENLP_dir)  # wait a few minutes...\n",
    "\n",
    "import cPickle\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the positive and negative dataset from Google to create sub-examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_pos = cPickle.load( open( \"data_positive.pickle\", \"rb\" ) )\n",
    "data_neg = cPickle.load( open( \"data_negative.pickle\", \"rb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'organisation': u'University of Paris', 'name': u'Charles Mantoux', 'value': 1, 'sentence': u'Charles Mantoux graduated from the University of Paris where Charles Mantoux studied under Broca.'}\n",
      "{'organisation': u'University of Michigan', 'name': u'James Robinson', 'value': 0, 'sentence': u'Robinson averaged 14.3 points and 5.7 assists during his collegiate career at the University of Michigan.'}\n",
      "911\n",
      "501\n"
     ]
    }
   ],
   "source": [
    "print data_pos[0]\n",
    "print data_neg[0]\n",
    "print len(data_pos)\n",
    "print len(data_neg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import makeParseTree\n",
    "\n",
    "sentence_positive = [data_pos[i]['sentence'] for i in range(len(data_pos))]\n",
    "sentence_negative = [data_neg[i]['sentence'] for i in range(len(data_neg))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "174\n",
      "Charles Mantoux graduated from the University of Paris where Charles Mantoux studied under Broca.\n",
      "------------------------------------------\n",
      "259\n",
      "Ralph Tatham graduated at the University of Cambridge in 1803.\n",
      "Seishi Kikuchi was born and grew up in Tokyo.\n",
      "Casteel earned bachelor's and master's degrees from California University (Pa.).\n",
      "Avgust Pirjevec studied Slavic philology at the University of Vienna.\n",
      "James P. McGranery graduated from Temple University Law School in 1928.\n",
      "Richter is a graduate of Skidmore College.\n",
      "Walter Erle matriculated at Queen's College, Oxford on 22 January 1602 aged 15.\n",
      "Brian Woermann attended Milford High School, where Matt Stryker played basketball and Diving.\n",
      "Stanley Holmes, 1st Baron Dovercourt was educated at the City of London School.\n",
      "Paul Leonard graduated from the Ohio University with a B.A.\n",
      "Cockburn was educated at Glenalmond College, Perthshire, and Worcester College, Oxford.\n",
      "Born in the Bronx, Spiro attended Xavier High School in Manhattan.\n",
      "Carl Epting Mundy, Jr. graduated from Sidney Lanier High School in Montgomery, Alabama.\n",
      "Prudhomme attended the Lycée Bonaparte, but eye trouble interrupted his studies.\n",
      "Sigurd Swane studied in Copenhagen at the Royal Danish Academy of Art.\n",
      "Medved attended Palisades High School when the family moved to Los Angeles, California.\n",
      "J. W. Crowdus attended St. Louis Medical College at St. Louis University.\n",
      "Gheorghe Ghimpu completed his studies at T. G. Shevchenko University in Tiraspol.\n",
      "Cledan Mears was educated at the University of Wales in Aberystwyth.\n",
      "Amos Degani studied public administration at the Hebrew University of Jerusalem.\n",
      "Before signing for Leeds United Alan attended Summerhill College in Sligo.\n",
      "Gemma Chan attended Newstead Wood School for Girls.\n",
      "Rychlik received his Ph.D. in 1980 from the Polish Academy of Sciences.\n",
      "John C. Lodge attended the Michigan Military Academy in 1881.\n",
      "Mitieli Bulanauca graduated from the University of New England, Australia (UNE) in 1996.\n",
      "Derek Cox played college football at William & Mary.\n",
      "Anthony Cotrone played high school football at Holy Trinity High School.\n",
      "Ali Asad Chandia attended Watkins Mill High School in Gaithersburg, Maryland.\n",
      "Shipman attended Worthington High School in Worthington, Ohio, and graduated in 1980.\n",
      "In 1977, Simmons received a Bachelor of Arts Degree from Texas State University.\n",
      "Iman Humaydan Younes studied sociology at the American University in Beirut.\n",
      "Lubetzky is a graduate of Trinity University and earned his J.D.\n",
      "Steve Bower was educated at Calday Grange Grammar School.\n",
      "George Mills attended the RAF Staff College in 1935.\n",
      "Laidig graduated from Minnetonka High School in Minnetonka in 1966.\n",
      "Nance graduated from the Phillips Exeter Academy in 1950.\n",
      "Robertson attended the University of Miami.\n",
      "Michael Geppi attended North Harford High School, graduating in 1993.\n",
      "Ísak Harðarson graduated from the University of Iceland in 1977.\n",
      "Frank Greenaway studied Chemistry at Jesus College, Oxford.\n",
      "Dr. Bajt attended the University of Dayton for two years.\n",
      "James McLachlan was graduated from Hamilton College, Clinton, New York, in 1878.\n",
      "Marcella Pobbe also studied at the Accademia Chigiana in Siena with Giorgio Favaretto.\n",
      "Lawrence Stenhouse was educated at St Andrews University in Scotland.\n",
      "Leach graduated from Leesville High School in 1951.\n",
      "Thomas Watson was educated at Winchester College and Oxford University.\n",
      "Amos K. Hadley graduated from Hamilton College.\n",
      "Yu received his Ph.D. from Princeton University's Department of Religion in 2008.\n",
      "Hugh Barker was educated at New College, Oxford.\n",
      "Ian attended Burlington Central High School from 1968 to 1971.\n",
      "Larry Hart also played at Holmes Community College.\n",
      "Abel Buckley was educated at Mill Hill School and Owen's College.\n",
      "William Miller attended Louisiana College.\n",
      "Torsten Oltmanns studied Economics at the University of Cologne (Germany).\n",
      "Jonathan Eig attended Northwestern University's Medill School of Journalism, graduating in 1986.\n",
      "Mykolas Biržiška graduated from law school at the University of Moscow in 1907.\n",
      "Born in Hawaii of Japanese ancestry, Shigeta studied drama at New York University.\n",
      "Walker attended Austin High School in Austin, Texas, graduating in 1999.\n",
      "Pelayo was educated at Far Eastern University in Manila, Philippines.\n",
      "Atkinson attended Presbyterian Ladies' College, Perth.\n",
      "Daniel Katz is a graduate of the University of Helsinki in the Humanities.\n",
      "Nirmal Ranjith Dewasiri read his PhD from Leiden University in The Netherlands.\n",
      "Jendrick graduated from Emerald Ridge High School in Puyallup, Washington.\n",
      "Reinhold and Ruth Benesch graduated from Leeds University in 1941.\n",
      "Judy Kuhn attended Georgetown Day School in Washington, D.C.\n",
      "Dunphy attended Malvern Preparatory School in Malvern, Pennsylvania.\n",
      "At the age of fourteen, Charles attended the University of Vermont, studying chemistry.\n",
      "Richard Pasco was educated at the King's College School, Wimbledon.\n",
      "Sanders attended college at Vanderbilt University.\n",
      "Robert Schwentke was a graduate of Columbia College Hollywood in 1992.\n",
      "Reinhardt graduated from University High School in Los Angeles.\n",
      "Kevin Baron graduated from the University of Richmond in 1997, with a B.A.\n",
      "Gourlay was educated at Kirkcaldy High School and was a vehicles examiner.\n",
      "Rabbi Klass received Smicha (Rabbinical Ordination) from Yeshiva Torah Vodaas .\n",
      "Kenneth Campbell was educated at Sedbergh School and Clare College, Cambridge\n",
      "Antony Hewish attended King's College, Taunton.\n",
      "Zeisler graduated from Northwestern University Law School in 1884.\n",
      "Harry Escott studied music at The Royal College of Music and Oxford University.\n",
      "In 1973 Heffermehl was hired as a sub-director under the Norwegian Consumer Ombudsman.\n",
      "Born in London, Richard Causton attended Quintin Kynaston School and William Ellis Schools.\n",
      "Mehmet Özhaseki completed his education at the Istanbul University Faculty of Law.\n",
      "In 1967, Fatima Hajaig earned her LLB from Eötvös Loránd University in Hungary.\n",
      "Rhodes graduated from the Command and General Staff College in 1908.\n",
      "Marijon Ancich attended San Pedro High School, graduating in 1955.\n",
      "His family moved to Palm Beach County in his childhood.\n",
      "From 1956-1958 Lal Mani Joshi studied at the University of Allahabad for B.A.\n",
      "Stromme graduated from Luther College (Iowa) and attended Concordia Theological Seminary.\n",
      "Bekian attended Venice High School and graduated in 1980.\n",
      "Rıza had graduated from Galatasaray High School in Constantinople.\n",
      "Georgia Frost attended John Colet School in Wendover.\n",
      "Patrick Young was educated at the University of St. Andrews (M.A., 1603).\n",
      "Simon Bainbridge was educated at Highgate School and the Royal College of Music.\n",
      "In 1959, John completed a Bachelor of Laws at University of Sydney.\n",
      "Ian McKay also holds a Masters degree from the University of Warwick.\n",
      "Mitro Repo studied theology at the University of Helsinki.\n",
      "Gerran Walker played college football at Lehigh.\n",
      "Born in Greenwood, Mississippi, Franklin graduated from Greenwood High School, Greenwood, Mississippi.\n",
      "Donald G. Bloesch received his undergraduate degree from Elmhurst College in Elmhurst, Illinois.\n",
      "Pierre Audi studied history at Exeter College, Oxford.\n",
      "Kristi Lee attended Indiana University.\n",
      "A native of Como, Argeo Quadri graduated from the Milan Conservatory in 1933.\n",
      "Caleb King attended Parkview High School in Lilburn, Georgia.\n",
      "Gary Collins studied economics and political science at Simon Fraser University.\n",
      "Carl Winchester studied at Corpus Christi College, Belfast.\n",
      "Lewis attended the State University of New York (SUNY) at Morrisville.\n",
      "Arnold was educated at Dalhousie University and ordained in 1938.\n",
      "Rhian Dodds attended Robert Morris University, playing for the Colonials before joining Kilmarnock.\n",
      "McLaughlin attended Elyria Catholic High School in Elyria, Ohio.\n",
      "Barwin attended the University of Detroit Jesuit High School.\n",
      "Newcombe earned a B.Sc from McMaster University in 1945.\n",
      "Phil Porter is a graduate of the University of Birmingham.\n",
      "Medoff received his PhD from Yeshiva University in 1991.\n",
      "Robert Webb was educated at Queen Elizabeth's Grammar School in Horncastle.\n",
      "Eric studied at the Slade School of Art.\n",
      "Phil Whelan holds a degree from the University of East Anglia.\n",
      "Born in San Francisco, California, Conrad graduated from Taft School in Watertown, Connecticut.\n",
      "Miyako Miyazaki has earned her Bachelors of Arts degree from the Kumamoto University.\n",
      "Donner studied English literature at Exeter College, Oxford and later entered business.\n",
      "Mike Mazurki attended La Salle Institute in Troy, New York, for high school.\n",
      "Bounds graduated with honors from the University of Virginia.\n",
      "Philip Moore attended Maidstone Grammar School.\n",
      "Roper was educated at Keble College, Oxford.\n",
      "Peter Cushman Jones was educated at the Boston Latin School in 1849.\n",
      "Ryan Amoroso attended Burnsville Senior High School in Burnsville, Minnesota.\n",
      "Senn graduated from Beaverton High School in 2003.\n",
      "Alan Solomont received an honorary degree from the University of Massachusetts Lowell.\n",
      "James Colquhoun entered St John's College, Cambridge in 1797 but left in 1798.\n",
      "Wrighton studied at the University of Warwick in the late 1990s.\n",
      "Sean Mackin attended Douglas Anderson School of the Arts.\n",
      "Born in Thomaston, Maine, Watts received his bachelors degree from Bowdoin College.\n",
      "Kelly also attended Doan's Hollow Public School in Port Dover.\n",
      "Deborah Walley attended Central High School in Bridgeport.\n",
      "Candy Devine was educated at Lourdes Hill College in Brisbane.\n",
      "Sue C. Payton was educated at Eastern Illinois University, receiving a B.S.\n",
      "Dietrichson was educated at Oslo National Academy of the Arts (2000-2004).\n",
      "Richards did undergraduate studies at the University of Utah.\n",
      "Nishimura graduated from Hanazono University from their Department of Buddhist Studies in 1956.\n",
      "Chris Lilik attended Villanova University and law school at Duquesne University.\n",
      "Cui Peng graduated from Beijing Film Academy.\n",
      "Evans graduated from Cottonwood High School in Salt Lake City.\n",
      "Danny Vasquez later played college soccer at Florida International University.\n",
      "Harold Koplow then enrolled at Tufts University for a graduate degree in Physics.\n",
      "Mendte graduated from Monsignor Bonner High School in nearby Drexel Hill.\n",
      "Leahy graduated from the United States Naval Academy in 1927.\n",
      "Yin graduated from China's Central Academy of Fine Arts in 2004.\n",
      "Victor E. Engstrom graduated from Lehigh University in 1936.\n",
      "Sarah Stevenson attended the Don Valley High School in Scawthorpe, north Doncaster.\n",
      "James Bowen Everhart studied law at Harvard University and in Philadelphia, Pennsylvania.\n",
      "George D. Archibald graduated from Washington & Jefferson College.\n",
      "Alex Boone played high school football at St. Edward High School in Lakewood.\n",
      "David Janssen attended Fairfax High School in Los Angeles.\n",
      "Ripia attended Rotorua Boys' High School.\n",
      "After college, Peter studied from 1989 at the Royal Conservatory of The Hague.\n",
      "Henry Cecil Kennedy Wyld studied at Charterhouse from 1883 to 1885.\n",
      "Robert Hill was educated at Christ's College, Cambridge, and graduated B.A.\n",
      "Prior to his professional career, Dibert played college football at Swarthmore College.\n",
      "William Hamilton Fyfe attended Fettes College in Edinburgh, Scotland.\n",
      "Victor Hettigoda received his education at Rahula College, Matara.\n",
      "From 1948 to 1949, Fall studied at the University of Paris.\n",
      "Luckmann studied sociology at the University of Vienna and Innsbruck.\n",
      "Gedeon attended William Cullen Bryant High School.\n",
      "Robert Grey Bushong attended Phillips Academy in Andover, Massachusetts.\n",
      "David A. Levinson is a 1997 graduate of Emerson College in Boston.\n",
      "Anderson graduated from Washington High School in Two Rivers, Wisconsin.\n",
      "Weston was educated at Blundell's School, Eton College and Exeter College, Oxford.\n",
      "Arthur Young was educated at Pembroke Hall, Cambridge, graduating LL.B.\n",
      "Jonathan Schanzer is a graduate of Lower Merion High School.\n",
      "Butler graduated from Lake Forest College in Lake Forest, Illinois.\n",
      "Smith attended Dunmore High School and Keystone College.\n",
      "Grove attended York College and graduated with a degree in public administration.\n",
      "After attending MCI, Daniel DeSanto then attended Ryerson University for Film Studies.\n",
      "Nigel Lockyer was educated at York University in Toronto, Canada.\n",
      "Meanwhile, his elder brother, Sir William Kirby Laing, studied engineering at Cambridge University.\n",
      "Pierre Russell attended Wyandotte High School.\n",
      "William Leigh was educated at Harrow and Magdalen College, Cambridge.\n",
      "Timothy Wu attended the Singapore American School, an international school in Singapore.\n",
      "Catherine R. Kinney has received honorary degrees from Georgetown University and Rosemont College.\n",
      "John W. Tyson graduated from Southern Methodist University, Dallas, TX.\n",
      "Nehemiah Donnellan graduated from Trinity College Dublin in 1666.\n",
      "Jérôme Colinet then played for Eupen at Belgian Second Division.\n",
      "From 1968 to 1969, Neal taught at the City College of New York.\n",
      "Saadi studied law at the University of Buenos Aires, graduating in 1977.\n",
      "Coleman Barks received an honorary doctorate from Tehran University in 2006.\n",
      "Jesse McNally is a Sunnydale High student played by Eric Balfour.\n",
      "Bell attended McGill University, Montreal, and studied under John William Dawson.\n",
      "McCallum attended Allan Glen's School in Glasgow to pursue studies in chemistry.\n",
      "Esherick is a graduate of Harvard College (1964, summa cum laude).\n",
      "Prior to playing football professionally, Joe played college football at Boston College University.\n",
      "Chiyo Okumura graduated from Aoyama Gakuin University.\n",
      "William Thomas graduated from Charlotte Hall Military Academy in 1804.\n",
      "Born in Platteville, Wisconsin, Estabrook graduated from the now University of Wisconsin-- Platteville.\n",
      "Buskirk graduated from Indiana University in Bloomington in 1841.\n",
      "Downs is a Phi Beta Kappa graduate of Indiana University.\n",
      "George V then attended Sandroyd School in Wiltshire, England.\n",
      "Donald Sinden was educated at Edgeborough and Lancing College.\n",
      "Sibling later attended Florida College.\n",
      "Cesare Dandini enrolled in 1621 in the Accademia del Disegno.\n",
      "John Speed entered Merchant Taylors' School in September, 1640.\n",
      "Christopher Daniels played college basketball at the University of Kentucky.\n",
      "Melin did his conscription at Norrlands dragonregemente (K4) in Umeå.\n",
      "Peter Lampl studied at Corpus Christi College, Oxford and the London Business School.\n",
      "George Hose was educated at St John's College, Cambridge and ordained in 1861.\n",
      "This network was called the ``Stanford University Network'' or SUN.\n",
      "From age 14, Alexander Sverjensky studied at the Petrograd Conservatory under Alexander Glazunov.\n",
      "Isaac attended Harrow School, and played for them against Eton College in 1917.\n",
      "Eric McCutchan was educated at Wesley College, Melbourne.\n",
      "Moss won the 1970, 1971, and 1974 World Series of Poker main events.\n",
      "John Warnock also has an honorary degree from the American Film Institute.\n",
      "Boyle previously received honorary doctorates from Napier University and the University of Aberdeen.\n",
      "King Yao graduated from Stuyvesant High School in New York, NY in 1989.\n",
      "Ben Cayenne graduated from the University of Cambridge in England.\n",
      "Bill Thom attended Marian Catholic High School in San Diego, California.\n",
      "In 1872 Johann Dzierzon received an honorary doctorate from the University of Munich.\n",
      "Plunket was educated at Eton College and served in the Army.\n",
      "Gail Galloway Adams taught at West Virginia University, retiring in 2008.\n",
      "In 1986, Gary A. Robbins began his career at the University of Connecticut.\n",
      "Martin Middlebrook entered National Service in 1950.\n",
      "Sydney B. Mitchell received an honorary Doctor of Literature degree from Occidental College.\n",
      "Russell Howland studied at the University of Illinois.\n",
      "Kelly Morgan attended Littleover Community School in Derby.\n",
      "Loel Guinness was educated at Royal Military College, Sandhurst.\n",
      "Louis Berger studied architecture at the Pratt Institute.\n",
      "After that experience, Pfister enrolled at the American Film Institute.\n",
      "Barker is a Juilliard Drama School graduate from St. Louis, Missouri.\n",
      "Gilbert was educated at Merchant Taylors' School and St John's College, Oxford.\n",
      "Augustin Pyramus de Candolle played college football at Howard University in Washington, DC.\n",
      "Arseny Sokolov graduated from Tomsk State University (TSU) in 1931.\n",
      "Suna Murray graduated from the Weston High School, Weston, Massachusetts, in 2007.\n",
      "William Alfred Browne received his university education at Queens College Galway.\n",
      "Brenda Paik Sunoo graduated from University of California, Los Angeles with a B.A.\n",
      "From 1950 to 1963, Jiří Levý lectured at Palacký University of Olomouc.\n",
      "William was educated at Westminster School and at Christ Church, Oxford.\n",
      "Tycho Brahe entered Trinity College, Cambridge in 1573, and graduated B.A.\n",
      "Thomas Best was educated at University College, Oxford.\n",
      "Thomas Carlyle studied at Bowdoin College for just one year from 1845.\n",
      "Kris Blanks finished in second place at the 2010 Puerto Rico Open.\n",
      "Endicott Peabody was graduated from Trinity College, Cambridge, in 1880 with an LL.B.\n",
      "Pradeep Kar is an alumnus of the Visvesvaraya National Institute of Technology, Nagpur.\n",
      "After this, Murtuza Mukhtarov continued his education at the Sorbonne University in Paris.\n",
      "Dunsany also received an honorary doctorate from Trinity College Dublin.\n",
      "Redmond was educated at Clongowes Wood College and Trinity College Dublin.\n",
      "Garland graduated first in his class from Harvard College with an A.B.\n",
      "Rosario is a former prop player at the Bicycle Casino.\n",
      "Zbigniew Bieńkowski returned to Poland during the Nazi occupation.\n",
      "Esherick taught at the University of Oregon before coming to UCSD in 1990.\n",
      "In addition, Senator Theodore Bilbo attended both Peabody College and the Law School.\n",
      "Leon Trotsky was a graduate of the University of St Petersburg.\n",
      "Lindsay graduated from Polytechnic High School in Riverside in 2003.\n",
      "Tom Howard attended Campion High School in Prairie du Chien, Wisconsin.\n",
      "Elias Lönnrot studied medicine at the Academy of Turku.\n",
      "Panah Huseynov graduated with a degree in History from the Azerbaijan State University.\n",
      "Henry Y. Satterlee graduated from Union College in 1829.\n",
      "Francis Goodwin then studied at the Royal Academy School.\n",
      "In 1998, Ashley Adams set a world record at the English Match.\n",
      "A. Katherine Grieb has also taught at Bangor Theological Seminary in Maine.\n",
      "In 1977, Ronald Reagan attended the University of South Florida.\n",
      "Wai was the child of a Native Hawaiian mother and a Chinese father.\n",
      "Stanley Moss was educated at Trinity College and Yale University.\n",
      "Young was educated at Winchester College.\n"
     ]
    }
   ],
   "source": [
    "# Filtering the sentences with more than 14 words. Parsing problems!\n",
    "final_dataset = []\n",
    "total_sentence_counter = 0\n",
    "for k in range(len(sentence_positive)):\n",
    "    if len(sentence_positive[k].split() )< 14 and sentence_positive[k][0] == sentence_positive[k][0].upper():\n",
    "#         print sentence_positive[k]\n",
    "        total_sentence_counter+=1\n",
    "        final_dataset.append(sentence_positive[k])\n",
    "print total_sentence_counter\n",
    "sent1 = sentence_positive[0]\n",
    "print sent1\n",
    "\n",
    "print \"------------------------------------------\"\n",
    "\n",
    "for k in range(len(sentence_negative)):\n",
    "    if len(sentence_negative[k].split() ) < 14 and sentence_negative[k][0] == sentence_negative[k][0].upper():\n",
    "#         print sent_n[k]\n",
    "        total_sentence_counter+=1\n",
    "        final_dataset.append(sentence_negative[k])\n",
    "print total_sentence_counter\n",
    "\n",
    "\n",
    "\n",
    "for sentence in final_dataset:\n",
    "    print sentence\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from makeParseTree import getParsedOutput , assignRoles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parse sentences to create subExamples using common ancestor of NP as outlined in paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "259\n",
      "0            3\n",
      "1            0\n",
      "3            4\n",
      "4            2\n",
      "5            4\n",
      "passed  6 Walter Erle matriculated at Queen's College, Oxford on 22 January 1602 aged 15.\n",
      "6            0\n",
      "7            6\n",
      "9            3\n",
      "10            6\n",
      "11            4\n",
      "12            2\n",
      "13            0\n",
      "14            3\n",
      "16            4\n",
      "17            4\n",
      "18            3\n",
      "19            4\n",
      "20            6\n",
      "21            2\n",
      "22            4\n",
      "23            3\n",
      "25            4\n",
      "26            4\n",
      "27            3\n",
      "28            4\n",
      "30            4\n",
      "31            5\n",
      "32            1\n",
      "33            2\n",
      "34            4\n",
      "35            2\n",
      "36            2\n",
      "37            2\n",
      "38            3\n",
      "39            4\n",
      "passed  40 Dr. Bajt attended the University of Dayton for two years.\n",
      "40            0\n",
      "41            3\n",
      "42            0\n",
      "43            3\n",
      "44            3\n",
      "45            4\n",
      "46            1\n",
      "47            5\n",
      "48            3\n",
      "49            3\n",
      "50            1\n",
      "51            4\n",
      "52            1\n",
      "54            3\n",
      "55            6\n",
      "57            4\n",
      "58            3\n",
      "59            3\n",
      "61            4\n",
      "62            2\n",
      "63            4\n",
      "64            3\n",
      "65            3\n",
      "67            3\n",
      "68            3\n",
      "69            6\n",
      "70            3\n",
      "71            5\n",
      "72            4\n",
      "73            0\n",
      "74            4\n",
      "75            3\n",
      "76            2\n",
      "77            6\n",
      "78            0\n",
      "79            4\n",
      "80            4\n",
      "81            6\n",
      "82            6\n",
      "83            2\n",
      "84            0\n",
      "85            3\n",
      "86            6\n",
      "87            2\n",
      "88            3\n",
      "89            0\n",
      "91            6\n",
      "93            5\n",
      "94            4\n",
      "95            0\n",
      "96            4\n",
      "97            4\n",
      "98            5\n",
      "99            1\n",
      "100            5\n",
      "101            3\n",
      "102            6\n",
      "103            3\n",
      "105            2\n",
      "106            3\n",
      "107            3\n",
      "108            4\n",
      "109            4\n",
      "110            5\n",
      "111            4\n",
      "112            3\n",
      "113            2\n",
      "115            5\n",
      "119            5\n",
      "120            1\n",
      "121            3\n",
      "122            3\n",
      "123            3\n",
      "124            3\n",
      "126            6\n",
      "127            4\n",
      "128            4\n",
      "129            5\n",
      "130            4\n",
      "131            3\n",
      "132            4\n",
      "133            3\n",
      "134            4\n",
      "135            4\n",
      "136            6\n",
      "138            1\n",
      "139            3\n",
      "140            4\n",
      "141            6\n",
      "142            2\n",
      "143            0\n",
      "145            2\n",
      "146            3\n",
      "147            4\n",
      "148            1\n",
      "149            6\n",
      "150            3\n",
      "151            1\n",
      "152            0\n",
      "153            3\n",
      "154            6\n",
      "156            3\n",
      "157            5\n",
      "passed  158 From 1948 to 1949, Fall studied at the University of Paris.\n",
      "158            0\n",
      "160            2\n",
      "161            3\n",
      "162            6\n",
      "163            5\n",
      "164            6\n",
      "165            0\n",
      "166            4\n",
      "167            4\n",
      "168            4\n",
      "169            6\n",
      "170            6\n",
      "171            3\n",
      "172            6\n",
      "173            1\n",
      "174            3\n",
      "175            6\n",
      "177            3\n",
      "178            3\n",
      "179            0\n",
      "180            4\n",
      "182            4\n",
      "183            0\n",
      "184            4\n",
      "185            0\n",
      "188            1\n",
      "189            2\n",
      "191            4\n",
      "192            4\n",
      "193            3\n",
      "194            2\n",
      "195            2\n",
      "196            0\n",
      "passed  197 John Speed entered Merchant Taylors' School in September, 1640.\n",
      "197            0\n",
      "198            5\n",
      "199            0\n",
      "200            6\n",
      "201            4\n",
      "202            3\n",
      "203            4\n",
      "205            3\n",
      "passed  206 Moss won the 1970, 1971, and 1974 World Series of Poker main events.\n",
      "206            0\n",
      "207            4\n",
      "209            6\n",
      "210            3\n",
      "211            5\n",
      "212            5\n",
      "213            2\n",
      "214            3\n",
      "215            5\n",
      "216            3\n",
      "217            6\n",
      "218            2\n",
      "219            3\n",
      "220            3\n",
      "221            3\n",
      "222            3\n",
      "223            6\n",
      "225            5\n",
      "226            4\n",
      "227            2\n",
      "228            4\n",
      "229            6\n",
      "230            3\n",
      "231            6\n",
      "232            6\n",
      "233            4\n",
      "passed  234 Thomas Carlyle studied at Bowdoin College for just one year from 1845.\n",
      "234            0\n",
      "235            0\n",
      "238            5\n",
      "239            3\n",
      "240            4\n",
      "241            5\n",
      "242            0\n",
      "243            0\n",
      "244            6\n",
      "247            4\n",
      "248            5\n",
      "249            0\n",
      "250            6\n",
      "251            2\n",
      "252            1\n",
      "253            0\n",
      "254            4\n",
      "255            6\n",
      "256            0\n",
      "257            4\n",
      "258            2\n",
      "-------------------------------\n",
      "736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/asus/misc/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:29: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n"
     ]
    }
   ],
   "source": [
    "print len(final_dataset)\n",
    "\n",
    "def for_each_example(sent):\n",
    "    node,adjacency,role = makeTree(getParsedOutput(sent,CORENLP))\n",
    "#     for i in range(len(node)):\n",
    "#         print i, node[i].Type , node[i].Text , role[i] , adjacency[i] \n",
    "    sol = assignRoles(node,adjacency,role)\n",
    "    return sol\n",
    "    print len(sol)\n",
    "\n",
    "ultimate_data = []\n",
    "for index ,sentence in enumerate(final_dataset):\n",
    "    num_sub_examples = 0\n",
    "    try:\n",
    "        parsed_output = for_each_example(sentence)\n",
    "        num_sub_examples = len(parsed_output)\n",
    "        if num_sub_examples > 6 :\n",
    "            continue\n",
    "        ultimate_data+= parsed_output\n",
    "    except:\n",
    "        print \"passed \", index , sentence\n",
    "        pass\n",
    "    print index , \"          \" , num_sub_examples\n",
    "\n",
    "print \"-------------------------------\"\n",
    "print len(ultimate_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "736\n",
      "dataset generated\n"
     ]
    }
   ],
   "source": [
    "print len(ultimate_data)\n",
    "\n",
    "import cPickle\n",
    "f = open('final_list_of_subexamples.pickle','w')\n",
    "cPickle.dump(ultimate_data,f)\n",
    "f.close()\n",
    "print \"dataset generated\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# f = open('sub_1_final_list_of_subexamples.pickle','w')\n",
    "# cPickle.dump(ultimate_data[:250],f)\n",
    "# f.close()\n",
    "# f = open('sub_2_final_list_of_subexamples.pickle','w')\n",
    "# cPickle.dump(ultimate_data[250:500],f)\n",
    "# f.close()\n",
    "# f = open('sub_3_final_list_of_subexamples.pickle','w')\n",
    "# cPickle.dump(ultimate_data[500:],f)\n",
    "# f.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
